import requests
from bs4 import BeautifulSoup
import pandas as pd
import time
import random

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
    'Connection': 'keep-alive',
    'Referer': 'https://www.douban.com/',
    'DNT': '1'  # 不要追踪
}

def get_douban_top250():
    # base_url = "https://movie.douban.com/top250"
    # https://movie.douban.com/chart
        movies = []
        for i in range(10):
            url = 'https://movie.douban.com/top250?start=' + str(i * 25) + '&filter='
            # print(url)
            try:
                # 发送HTTP请求
                response = requests.get(url, headers=headers, timeout=10)
                response.raise_for_status()  # 检查请求是否成功

                # 添加随机延迟，避免请求过快
                time.sleep(random.uniform(1, 3))

                # 解析HTML内容
                soup = BeautifulSoup(response.text, 'html.parser')

                # 查找所有电影项
                movie_list = soup.find('ol', class_='grid_view').find_all('li')

                if not movie_list:
                    print(f"第 {i} 页未找到电影数据")
                    continue

                print(f"正在处理第 {i} 页，共 {len(movie_list)} 部电影")
                # print(movie_list[0])

                for movie in movie_list:
                    movie_info = {}

                    #获取电影排名
                    movie_rank = movie.find('em').get_text()
                    movie_info['movie_rank'] = movie_rank

                    # 获取电影标题
                    movie_title = movie.find('div', class_='hd').find('span', class_='title').get_text()
                    movie_info['title'] = movie_title

                    # 获取电影评分
                    movie_rating_num = movie.find('span', class_='rating_num').get_text()
                    movie_info['movie_rating_num'] = movie_rating_num

                    # 获取电影评论人数
                    movie_comment_num = movie.find('span', string=lambda text: text and '人评价' in text).get_text()
                    movie_info['movie_comment_num'] = movie_comment_num

                    # 获取电影描述
                    quote = movie.find('p', class_='quote')
                    if quote:
                        movie_describe = movie.find('p', class_='quote').find('span').get_text()
                    else:
                        movie_describe = ''
                    movie_info['movie_describe'] = movie_describe

                    # 获取电影详情
                    # 获取br标签后的数据 next_sibling
                    # 获取br标签前的数据 previous_sibling
                    movie_details = movie.find('div', class_='bd').find('p').find('br').next_sibling.strip()
                    # print(movie_details.replace(" / ", "/"))
                    # 去除’/‘前后符号
                    movie_info['movie_details'] = movie_details.replace(" / ","/")

                    # 获取电影链接
                    movie_url = movie.find('div', class_='pic').find('a')['href']
                    movie_info['movie_url'] = movie_url
                    # print(movie_info)
                    movies.append(movie_info)

            except requests.exceptions.RequestException as e:
                print(f"请求第 {i} 页失败: {e}")
                continue
        return movies




def save_to_csv(movies, filename='douban_top250.csv'):
    """保存数据到CSV文件"""
    # 判断是否有数据可保存
    if not movies:
        print("没有数据可保存")
        return False
    # 将数据转换为DataFrame
    df = pd.DataFrame(movies)
    # 调整列顺序
    columns = ['movie_rank', 'title', 'movie_rating_num', 'movie_comment_num', 'movie_describe',
               'movie_details', 'movie_url']
    df = df[columns]
    # 保存到CSV
    df.to_csv(filename, index=False, encoding='utf-8-sig')
    print(f"数据已保存到 {filename}")
    return True



def main():
    """主函数"""
    print("开始获取豆瓣电影Top250数据...")
    movies = get_douban_top250()

    if not movies:
        print("未能获取电影数据，程序退出")
        return

    print(f"\n成功获取 {len(movies)} 部电影数据")

    # 保存到CSV
    save_to_csv(movies)


if __name__ == "__main__":
    main()